# coding:utf-8

import os
from common.config import config_option
from utils.spider_utils import get_soup_html, download_file
from utils.date_utils import get_dates_by_date


class ProbeData(object):
    """
    探测数据
    """

    def __init__(self):
        # 爬虫参数配置
        self.base_url = 'http://data.caida.org/datasets/topology/ark/ipv4/probe-data/'
        # self.team_list = ['team-1', 'team-2', 'team-3']
        # self.year_list = [str(i) for i in range(2007, 2019)]
        self.team_list = ['team-1']
        self.year_list = ['2019']
        self.date_list = ["cycle-" + i.replace("-", "") for i in get_dates_by_date('2019-01-01', '2019-09-03')]

        # 存储配置
        self.output = config_option['project_path'] + "/output/probe/"

    def get_page(self):
        """获取页面"""
        page_list = []

        for team in self.team_list:
            url_team = self.base_url + team + "/"
            for year in self.year_list:
                url_year = url_team + year + "/"
                for day in self.date_list:
                    url_date = url_year + day + "/"
                    page_list.append(url_date)
        return page_list

    def get_url(self, page_list):
        """获取URL"""
        url_list = []
        for page in page_list[:5]:
            print(page)
            soup = get_soup_html(page)
            tag_list = soup.findAll({"a"})
            for tag in tag_list:
                href = page + tag['href']
                if href.endswith(".warts.gz"):
                    url_list.append(href)
        print(url_list)
        return url_list

    def download_file(self, url_list):
        """下载文件"""
        for url in url_list:
            filename = os.path.basename(url)
            dirname = os.path.basename(os.path.dirname(url)).replace("cycle-", "")
            dir_path = self.output + dirname + os.sep
            download_file(dir_path, filename, url)

    def run(self):
        page_list = self.get_page()
        url_list = self.get_url(page_list)
        self.download_file(url_list)


if __name__ == "__main__":
    probe = ProbeData()
    probe.run()
